from python_ai.common.xcommon import *
from python_ai.DL.tensorflow.common.text_proc import *
import re


def filter_string(str):
    return re.sub(r'[\x00-\x7F]', '', str)


path = r'..\..\..\..\..\large_data\DL1\cnn_text\\'
is_test = False
encoding = 'utf8'
# errors = 'ignore'
errors = None
if is_test:
    xname = '2'
else:
    xname = ''
positive_data_file = path + 'rt-polarity' + xname + '.pos'  #正面评价文本
negative_data_file = path + 'rt-polarity' + xname + '.neg'  #负面评价文本
with open(positive_data_file, 'r', encoding=encoding, errors=errors) as f:
    pos_ex = f.readlines()
    sep(type(pos_ex))
    print(pos_ex[:5])
with open(negative_data_file, 'r', encoding=encoding, errors=errors) as f:
    neg_ex = f.readlines()
    sep(type(neg_ex))
    print(neg_ex[:5])

# splice pos and neg
sep('splice pos and neg')
x_sentences = pos_ex + neg_ex
# format sentences
x_sentences_filtered = [filter_string(sent) for sent in x_sentences]
x_sentences = []
for i, s in enumerate(x_sentences_filtered):
    if len(s) == 0:
        continue
    x_sentences.append(s + '_' + str(i))
print(x_sentences[:5])

with open('cnn_analysis_' + encoding + '_' + str(errors) + '.txt', 'w', encoding=encoding) as f:
    x_sentences_with_n = [sent + '\n' for sent in x_sentences]
    f.writelines(x_sentences_with_n)